<!DOCTYPE html>












  


<html class="theme-next mist use-motion" lang="zh-Hans">
<head><meta name="generator" content="Hexo 3.8.0">
  <meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">












<meta http-equiv="Cache-Control" content="no-transform">
<meta http-equiv="Cache-Control" content="no-siteapp">






















<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css">

<link href="/css/main.css?v=6.4.1" rel="stylesheet" type="text/css">


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=6.4.1">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=6.4.1">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=6.4.1">


  <link rel="mask-icon" href="/images/logo.svg?v=6.4.1" color="#222">









<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Mist',
    version: '6.4.1',
    sidebar: {"position":"left","display":"hide","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: false,
    fastclick: false,
    lazyload: false,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>


  




  <meta name="description" content="对Redis进行扩展">
<meta name="keywords" content="Redis,scalability">
<meta property="og:type" content="article">
<meta property="og:title" content="Redis水平扩展">
<meta property="og:url" content="https://shaotao1988.github.io/2018/09/09/Redis-Scaling/index.html">
<meta property="og:site_name" content="Tao&#39;s Blog">
<meta property="og:description" content="对Redis进行扩展">
<meta property="og:locale" content="zh-Hans">
<meta property="og:updated_time" content="2018-11-01T05:02:08.072Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Redis水平扩展">
<meta name="twitter:description" content="对Redis进行扩展">






  <link rel="canonical" href="https://shaotao1988.github.io/2018/09/09/Redis-Scaling/">



<script type="text/javascript" id="page.configurations">
  CONFIG.page = {
    sidebar: "",
  };
</script>

  <title>Redis水平扩展 | Tao's Blog</title>
  









  <noscript>
  <style type="text/css">
    .use-motion .motion-element,
    .use-motion .brand,
    .use-motion .menu-item,
    .sidebar-inner,
    .use-motion .post-block,
    .use-motion .pagination,
    .use-motion .comments,
    .use-motion .post-header,
    .use-motion .post-body,
    .use-motion .collection-title { opacity: initial; }

    .use-motion .logo,
    .use-motion .site-title,
    .use-motion .site-subtitle {
      opacity: initial;
      top: initial;
    }

    .use-motion {
      .logo-line-before i { left: initial; }
      .logo-line-after i { right: initial; }
    }
  </style>
</noscript>

</head>

<body itemscope="" itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope="" itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">Tao's Blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
    
  </div>

  <div class="site-nav-toggle">
    <button aria-label="Toggle navigation bar">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>



<nav class="site-nav">
  
    <ul id="menu" class="menu">
      
        
        
        
          
          <li class="menu-item menu-item-home">
    <a href="/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-home"></i> <br>Home</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-about">
    <a href="/about/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-user"></i> <br>About</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-tags">
    <a href="/tags/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-tags"></i> <br>Tags</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-archives">
    <a href="/archives/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-archive"></i> <br>Archives</a>
  </li>

      
      
    </ul>
  

  
    

  

  
</nav>



  



</div>
    </header>

    


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  

  <article class="post post-type-normal" itemscope="" itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://shaotao1988.github.io/2018/09/09/Redis-Scaling/">

    <span hidden itemprop="author" itemscope="" itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Shao Tao">
      <meta itemprop="description" content="Programming and Technical Sharing">
      <meta itemprop="image" content="/images/avatar.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope="" itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Tao's Blog">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">Redis水平扩展
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">

            
            
            

            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">Posted on</span>
              

              
                
              

              <time title="Created: 2018-09-09 00:00:00" itemprop="dateCreated datePublished" datetime="2018-09-09T00:00:00-05:00">2018-09-09</time>
            

            
              

              
                
                <span class="post-meta-divider">|</span>
                

                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                
                  <span class="post-meta-item-text">Edited on</span>
                
                <time title="Modified: 2018-11-01 00:02:08" itemprop="dateModified" datetime="2018-11-01T00:02:08-05:00">2018-11-01</time>
              
            
          </span>

          

          
            
          

          
          

          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <p>对Redis进行扩展</p>
<a id="more"></a>
<p>在进行扩展之前，我们需要审视应用是否可能存在如下方面的性能问题：</p>
<ul>
<li>如果使用了ziplist、intset等来降低内存用量，需要检查entries配置项是否过大</li>
<li>检查是否使用了最合适的存储类型，比如treat lists like sets; 将整个hash抓取过来再在代码中排序（可能使用zset在服务器端排序更合适）</li>
<li>如果需要发送很大的对象给Redis缓存，可以考虑先压缩</li>
<li>使用pipelining减少roundtrip</li>
<li>重用Redis连接，不要每次都去创建新的连接</li>
</ul>
<h2 id="Scaling-Read：增加Slave"><a href="#Scaling-Read：增加Slave" class="headerlink" title="Scaling Read：增加Slave"></a>Scaling Read：增加Slave</h2><ul>
<li>只能对Master做写操作</li>
<li>加密和压缩：如果CPU快但网络带宽小，使用压缩，如果带宽够大，可以不压缩</li>
<li>层次化Master-Slave拓扑，避免单个Master的网络带宽成为性能瓶颈</li>
</ul>
<h2 id="Scaling-Writes-and-memory-capcity-Sharding"><a href="#Scaling-Writes-and-memory-capcity-Sharding" class="headerlink" title="Scaling Writes and memory capcity: Sharding"></a>Scaling Writes and memory capcity: Sharding</h2><p>在扩展写性能之前需要审视的几件事：</p>
<ul>
<li>将相互无关联的数据放到不同的Redis实例上</li>
<li>使用pipelining节省roundtrip</li>
<li>在使用WATCH/MULTI/EXEC时，保证WATCH的范围不要太大，否则应考虑使用自定义的更精细的分布式锁</li>
<li>如果使用了AOF，需要保证足够的硬盘空间</li>
</ul>
<p><strong>通过sharding将数据存放到不同的服务器上</strong>，在存取数据时需要先使用跟sharding一致的路由算法找到数据所在的服务器。</p>
<p>Presharding for growth: 为未来扩容预留shards，开始可以将多个shards运行在同一台服务器上的多个Redis实例上，或者一个Redis的多个库中。有多个Redis实例时需要保证它们配置不同的端口和snapshot、AOF文件路径。</p>
<blockquote>
<p>建议将shard数设为2的n次方，并且在未来扩容时成倍的扩充服务器，这样只需要将原来服务器上一半的库迁移到新服务器上</p>
</blockquote>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">REDIS_CONNECTIONS = &#123;&#125;</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_redis_connection</span><span class="params">(component, wait = <span class="number">1</span>)</span>:</span></span><br><span class="line">    key = <span class="string">'config:reids:'</span> + component</span><br><span class="line">    old_config = CONFIGS.get(key, object()) <span class="comment"># 从配置文件获取老配置</span></span><br><span class="line">    config = get_config(config_connection, <span class="string">'redis'</span>, component, wait) <span class="comment"># Redis服务器获取新配置</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> config != old_config:</span><br><span class="line">        CONFIGS.set(key, config)</span><br><span class="line">        REDIS_CONNECTIONS[key] = redis.Redis(**config) <span class="comment"># 创建新连接并保存</span></span><br><span class="line">    </span><br><span class="line">    <span class="keyword">return</span> REDIS_CONNECTIONS[key]</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_sharded_connection</span><span class="params">(component, key, shard_count, wait)</span>:</span></span><br><span class="line">    shard = shard_key(component, <span class="string">'x'</span>+str(key), shard_count, <span class="number">2</span>)</span><br><span class="line">    <span class="keyword">return</span> get_redis_connection(shard, wait)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 装饰器，用于装饰所有需要连接Redis服务器的函数</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">sharded_connection</span><span class="params">(component, shard_count, wait = <span class="number">1</span>)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">wrapper</span><span class="params">(function)</span>:</span></span><br><span class="line"><span class="meta">        @functools.wraps(function)</span></span><br><span class="line">        <span class="function"><span class="keyword">def</span> <span class="title">call</span><span class="params">(conn, key, *args, **kwargs)</span>:</span></span><br><span class="line">            conn = get_sharded_connection(component, key, shard_count, wait)</span><br><span class="line">            <span class="keyword">return</span> function(conn, key, *args, **kwargs)</span><br><span class="line">        <span class="keyword">return</span> call</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="meta">@sharded_connection('unique', 16)</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">count_visit</span><span class="params">(conn, session_id)</span>:</span></span><br><span class="line">    ...</span><br><span class="line">    <span class="keyword">if</span> shard_add(conn, key, id, expected, SHARD_SIZE):</span><br><span class="line">        conn.incr(key)</span><br></pre></td></tr></table></figure>

      
    </div>

    

    
    
    

    

    

    

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/Redis/" rel="tag"># Redis</a>
          
            <a href="/tags/scalability/" rel="tag"># scalability</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2018/09/09/Redis-Transaction/" rel="next" title="Redis事务">
                <i class="fa fa-chevron-left"></i> Redis事务
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2018/09/09/Redis-Best-Practice/" rel="prev" title="Redis最佳实践">
                Redis最佳实践 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>


  </div>


          </div>
          

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            Table of Contents
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            Overview
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope="" itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image" src="/images/avatar.jpg" alt="Shao Tao">
            
              <p class="site-author-name" itemprop="name">Shao Tao</p>
              <p class="site-description motion-element" itemprop="description">Programming and Technical Sharing</p>
          </div>

          
            <nav class="site-state motion-element">
              
                <div class="site-state-item site-state-posts">
                
                  <a href="/archives/">
                
                    <span class="site-state-item-count">22</span>
                    <span class="site-state-item-name">posts</span>
                  </a>
                </div>
              

              

              
                
                
                <div class="site-state-item site-state-tags">
                  <a href="/tags/index.html">
                    
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                    <span class="site-state-item-count">15</span>
                    <span class="site-state-item-name">tags</span>
                  </a>
                </div>
              
            </nav>
          

          

          

          
          

          
          

          
            
          
          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#Scaling-Read：增加Slave"><span class="nav-number">1.</span> <span class="nav-text">Scaling Read：增加Slave</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Scaling-Writes-and-memory-capcity-Sharding"><span class="nav-number">2.</span> <span class="nav-text">Scaling Writes and memory capcity: Sharding</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2019</span>
  <span class="with-love" id="animate">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Shao Tao</span>

  

  
</div>




  <div class="powered-by">Powered by <a class="theme-link" target="_blank" href="https://hexo.io">Hexo</a> v3.8.0</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">Theme – <a class="theme-link" target="_blank" href="https://theme-next.org">NexT.Mist</a> v6.4.1</div>




        








        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

    
	
    

    
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>


























  
  
    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  


  


  <script type="text/javascript" src="/js/src/utils.js?v=6.4.1"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=6.4.1"></script>



  
  

  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=6.4.1"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=6.4.1"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=6.4.1"></script>



  



  










  





  

  

  

  

  
  

  

  

  

  

  

</body>
</html>
